#/usr/bin/env python
# -*- coding: UTF-8 -*-
import time
import requests
import pymysql
conf ={
	"host":"127.0.0.1",
	"user":"root",
	"password":"123456",
	"database":"test",
	"charset":"utf8"
}
def lagou(page,position,city):
		headers = {'Referer':'https://www.lagou.com/jobs/list_'+position+'?city=%E5%B9%BF%E5%B7%9E&cl=false&fromSearch=true&labelWords=&suginput=', 
				   'Origin':'https://www.lagou.com',                
				   'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36',
				   'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
				   'Cookie':'user_trace_token=20171208164311-5aca723d-93b4-4fdc-b7d9-a95ec905be5e; LGUID=20171208164313-d3826136-dbf3-11e7-9c6f-5254005c3644; _ga=GA1.2.1996200803.1512722673; _gid=GA1.2.553985016.1524739712; index_location_city=%E5%B9%BF%E5%B7%9E; WEBTJ-ID=20180427113341-1630529e57f79-0fe05f46e5b0c8-4545092c-2073600-1630529e58225f; _gat=1; LGSID=20180427113119-7343409a-49cb-11e8-bae7-5254005c3644; PRE_UTM=m_cf_cpc_baidu_pc; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Fbaidu.php%3Fsc.K60000aekzsCKhx2dE_QVgGkaZPir-dyFgMTdXSNRqTuBas3nP4YyYtr2nhjLjh_tUwh8twta39INrPtdTIA8Xlpfylx3Nd97Q6C57NXQ9omXrDr1NHTfctdly2vnTrsK3zztO-levkr6GEJrofhvQqEvya7pwHBXAoIT71_i1U2lJCorf.DY_NR2Ar5Od663rj6tJQrGvKD7ZZKNfYYmcgpIQC8xxKfYt_U_DY2yP5Qjo4mTT5QX1BsT8rZoG4XL6mEukmryZZjzl8Z-HgguoLtVviEGzF4T5MY3IMo9vUt5M8se5gjlSXZ1tT5ot_rSVHReiM-kl-9h9mlX5uYqB6.U1Yk0ZDqs2v4VnL30ZKGm1Yk0Zfqs2v4VnL30A-V5HcsP0KM5yF-TZns0ZNG5yF9pywd0ZKGujYY0APGujY3P0KVIjYknjDLg1DsnH-xnH0zndtknjfvg1nvnjD0pvbqn0KzIjYkn100uy-b5HcvnWRYg1DYn1NxnWDknWFxnWmkrjwxnWcsP1n0mhbqnW0Y0AdW5HTzrHmzP10zPdtLnWbvnWTsn1KxnNtknjFxn0KkTA-b5H00TyPGujYs0ZFMIA7M5H00mycqn7ts0ANzu1Ys0ZKs5H6vP1cLn1mvPfK8IM0qna3snj0snj0sn0KVIZ0qn0KbuAqs5H00ThCqn0KbugmqTAn0uMfqn0KspjYs0Aq15H00mMTqnH00UMfqn0K1XWY0IZN15HfvnjR3Pj6krjDvPHb1nHcYnW00ThNkIjYkPHcYP1b4rjTYrHRv0ZPGujdBnyDkmWf1rH0snj03nvf40AP1UHdDrHf1fW6LnHn3wHb3wb7D0A7W5HD0TA3qn0KkUgfqn0KkUgnqn0KlIjYs0AdWgvuzUvYqn7tsg1Kxn7ts0Aw9UMNBuNqsUA78pyw15HKxn7tsg1nkrjmvndts0ZK9I7qhUA7M5H00uAPGujYzPW6Yn1RYPHm0ugwGujYVnfK9TLKWm1Ys0ZNspy4Wm1Ys0Z7VuWYs0AuWIgfqn0KhXh6qn0Khmgfqn0KlTAkdT1Ys0A7buhk9u1Yk0Akhm1Ys0APzm1YznjTzP6%26ck%3D1724.2.102.273.561.296.559.402%26shh%3Dwww.baidu.com%26sht%3Dbaiduhome_pg%26us%3D1.0.1.0.1.315.0%26wd%3D%25E6%258B%2589%25E5%258B%25BE%25E7%25BD%2591%26issp%3D1%26f%3D8%26ie%3Dutf-8%26rqlang%3Dcn%26tn%3Dbaiduhome_pg%26inputT%3D2406%26bc%3D110101; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Flp%2Fhtml%2Fcommon.html%3Futm_source%3Dm_cf_cpc_baidu_pc%26m_kw%3Dbaidu_cpc_gz_e110f9_d2162e_%25E6%258B%2589%25E5%258B%25BE%25E7%25BD%2591; JSESSIONID=ABAAABAAADEAAFIA22B4EB2A23AC161FC35F5A2751646A6; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1524740678,1524740683,1524800022,1524800028; TG-TRACK-CODE=index_navigation; SEARCH_ID=a8f7fa4b419243fb82c13c2b049f2312; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1524800440; LGRID=20180427113817-6c680526-49cc-11e8-a5c8-525400f775ce'
				   }
		dates={'first':'true',
			   'pn': page,
			   'kd': position}
		url='https://www.lagou.com/jobs/positionAjax.json?city='+city+'&needAddtionalResult=false&isSchoolJob=0'
		resp = requests.post(url,data=dates,headers=headers)
		resp1 = resp.content.decode('utf-8')
		result=resp.json()['content']['positionResult']['result']
		db = pymysql.connect(**conf)
		timeNow = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
		cursor = db.cursor()
		for i in result:
			positionId =i['positionId']
			companyFullName =i['companyFullName']
			positionName =i['positionName']
			salary =i['salary']
			workYear =i['workYear']
			jobNature =i['jobNature']
			address =i['district']
			select = "select * from lagou where positionId='%s'"%(positionId)
			cursor.execute(select)
			results = cursor.fetchall()
			if len(results)==0:
				sql = "insert into lagou(positionId,companyFullName,positionName,salary,workYear,jobNature,district) values('%s','%s','%s','%s','%s','%s','%s')"%(positionId, companyFullName,positionName,salary,workYear,jobNature,address)
				cursor.execute(sql)
				db.commit()
									
			else:
				print('----数据已存在------'.decode("utf-8").encode("gbk"))
		cursor.close()		
		db.close()
def main(position,city):
            page = 1
            while page<=30:
				pages = bytes(page)
				tip ='---------------------第'+pages+'页--------------------'
				print(tip.decode("utf-8").encode("gbk"))
				lagou(page,position,city)
				page=page+1
#输入你想要爬取的职位名,如:python
if __name__ == '__main__':
		main('java','广州')